{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EtyN9XsZ245Q"
   },
   "source": [
    "# Working with Language Models and Tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "apmWcx6z23Z1"
   },
   "outputs": [],
   "source": [
    "!pip install transformers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 4956,
     "status": "ok",
     "timestamp": 1619015589798,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "-MQ_9wGWnXBd"
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer \n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4142,
     "status": "ok",
     "timestamp": 1619015589800,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "9YGDDnaMncAW",
    "outputId": "107fc474-7a0e-4d8e-d126-5242de5996ba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2478, 19081, 2003, 3733, 999, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"Using transformers is easy!\" \n",
    "tokenizer(text) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 1173,
     "status": "ok",
     "timestamp": 1619015591660,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "_wiigdlCnfQJ"
   },
   "outputs": [],
   "source": [
    "encoded_input = tokenizer(text, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 4150,
     "status": "ok",
     "timestamp": 1619015596662,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "Qm1YaZBMni6d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ee073709bd649cea11d73d69594e8f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fee41a4ae1fc43ddab7ed803246208ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertModel \n",
    "model = BertModel.from_pretrained(\"bert-base-uncased\") \n",
    "output = model(**encoded_input) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10456,
     "status": "ok",
     "timestamp": 1619015610514,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "wUSqcni1nsgL",
    "outputId": "daf1df39-d972-47b7-f465-f9a5b0c0fda0"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00e46639b88142bfbaecbfb475cf8fe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, TFBertModel \n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') \n",
    "model = TFBertModel.from_pretrained(\"bert-base-uncased\") \n",
    "text = \" Using transformers is easy!\" \n",
    "encoded_input = tokenizer(text, return_tensors='tf') \n",
    "output = model(**encoded_input) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5518,
     "status": "ok",
     "timestamp": 1619015632607,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "fIqIZ6-Jn3n8",
    "outputId": "c2d096e7-cc6b-4a10-886c-4035d7639682"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d7d5e3e4c9c4d7c93fdc897fe51d2dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'sequence': 'the man worked as a carpenter.',\n",
       "  'score': 0.0974755585193634,\n",
       "  'token': 10533,\n",
       "  'token_str': 'carpenter'},\n",
       " {'sequence': 'the man worked as a waiter.',\n",
       "  'score': 0.05238321051001549,\n",
       "  'token': 15610,\n",
       "  'token_str': 'waiter'},\n",
       " {'sequence': 'the man worked as a barber.',\n",
       "  'score': 0.04962703585624695,\n",
       "  'token': 13362,\n",
       "  'token_str': 'barber'},\n",
       " {'sequence': 'the man worked as a mechanic.',\n",
       "  'score': 0.03788604959845543,\n",
       "  'token': 15893,\n",
       "  'token_str': 'mechanic'},\n",
       " {'sequence': 'the man worked as a salesman.',\n",
       "  'score': 0.037680838257074356,\n",
       "  'token': 18968,\n",
       "  'token_str': 'salesman'}]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline \n",
    "unmasker = pipeline('fill-mask', model='bert-base-uncased') \n",
    "unmasker(\"The man worked as a [MASK].\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 34502,
     "status": "ok",
     "timestamp": 1619015684885,
     "user": {
      "displayName": "Savas Yıldırım",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64",
      "userId": "10717726124681851716"
     },
     "user_tz": -180
    },
    "id": "KNR6jL6uoDB5",
    "outputId": "09e9407e-0cd9-40d8-ed5d-de6176549c3c"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79daa210226044d2bba3394d4945e1fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1154.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecf39bec6ed1482b8596944715f55831",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1629486723.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at facebook/bart-large-mnli were not used when initializing BartModel: ['model.encoder.version', 'model.decoder.version']\n",
      "- This IS expected if you are initializing BartModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BartModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7743faf8dcdf4c0f88a2628fd5eb7a3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "655a2bbfb7374c08808e6d654a5a3582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33e76e79622c4cc8a3e8c5b8678ec915",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1355863.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at facebook/bart-large-mnli were not used when initializing BartForSequenceClassification: ['model.encoder.version', 'model.decoder.version']\n",
      "- This IS expected if you are initializing BartForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BartForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'sequence': 'I am going to france.',\n",
       " 'labels': ['travel', 'dancing', 'cooking'],\n",
       " 'scores': [0.9866883754730225, 0.007197574246674776, 0.006114062387496233]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline \n",
    "classifier = pipeline(\"zero-shot-classification\", model=\"facebook/bart-large-mnli\") \n",
    "sequence_to_classify = \"I am going to france.\" \n",
    "candidate_labels = ['travel', 'cooking', 'dancing'] \n",
    "classifier(sequence_to_classify, candidate_labels) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNA/NFmK6sxOzqVBmo0oq+k",
   "name": "CH02.01 Working with Language Models and Tokenizers .ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
